package youleche.com.common;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import youleche.com.business.BArea;
import youleche.com.factory.MLLogFactory;
import youleche.com.model.MArea;

public class MeilecheChinaAreaCrawler {	
	/**
	 * 没扫描的区域数据节点
	 */
	private static List<MArea> noScanAreaList = new LinkedList<MArea>();
	
	/**
	 * 获取未扫描的区域节点数据
	 * @return 未扫描的区域节点数据
	 */
	public static List<MArea> getNoScanAreaList(){
		isAddScanArea = true;
		return noScanAreaList;
	}
	
	/**
	 * 获取最后扫描的节点(第一个未扫描节点的上一个兄弟节点)
	 */
	private static MArea lastScanArea = null;
	
	/**
	 * 是否添加扫描
	 */
	private static boolean isAddScanArea = true;
	
	/**
	 * 获取最后扫描的节点(第一个未扫描节点的上一个兄弟节点)
	 * @return 最后扫描的节点(第一个未扫描节点的上一个兄弟节点)
	 */
	public static MArea getLastScanAreaList(){
		return lastScanArea;
	}
	
	private static MeilecheChinaAreaCrawler crawler = null;
	/**
	 * 获取美乐车爬虫实例
	 * @return 爬虫实例
	 */
	public static MeilecheChinaAreaCrawler getInstance(){
		if(crawler == null){
			crawler = new MeilecheChinaAreaCrawler();
		}
		
		return crawler;
	}
	
	/**
	 * 统计局数据
	 */
	private String statsUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/";

	/**
	 * 获取中国统计局数据网页的html码
	 * @param urlAddr url地址
	 * @return 返回网页源代码
	 * @throws Exception 异常
	 */
	@SuppressWarnings("static-access")
	public String get_www_stats_gov_cn_HtmlContent(String urlAddr) throws Exception {
		URL url = new URL(urlAddr);

		HttpURLConnection con = (HttpURLConnection) url.openConnection();
		con.setFollowRedirects(true);
		con.setInstanceFollowRedirects(false);
		con.connect();
		BufferedReader br = new BufferedReader(new InputStreamReader(
				con.getInputStream(), "gb2312"));
		String s = "";
		StringBuffer sb = new StringBuffer("");
		while ((s = br.readLine()) != null) {
			sb.append(s + "\r\n");
		}

		return sb.toString();
	}
	
	/**
	 * 
	 * @param urlAddr
	 */
	public List<String> parseProvinceAndCode(String urlAddr) throws Exception{
		List<String> list = new LinkedList<String>();
		String htmlContent = "";
		try{
			htmlContent = this.get_www_stats_gov_cn_HtmlContent(urlAddr);
		}
		catch(Exception e){
			htmlContent = this.get_www_stats_gov_cn_HtmlContent(urlAddr);
			MLLogFactory.exceptionLog("url:" + urlAddr + "---------parentCode:无"  + " -----------level:1" , "抓取数据失败", e.getMessage(), "localhost");
		}
		
		String regex = "<td><a href='\\d{0,}.html'>[\u4e00-\u9fa5]{0,}<br/></a></td>";
		Pattern pattern = Pattern.compile(regex);
		Matcher match = pattern.matcher(htmlContent);
		while(match.find()){
			String tdHtml = match.group();
			String codeRegex = "\\d{0,}.html";
			Pattern codePattern = Pattern.compile(codeRegex);
			Matcher codeMatch = codePattern.matcher(tdHtml);
			
			//code
			String code = "";
			while(codeMatch.find()){
				String urlCode = codeMatch.group(0);
				code = urlCode.split("\\.")[0];
				list.add(urlCode + "^" + code);
				break;
			}

			//name
			String nameRegex = "[\u4E00-\u9FA5]{0,}<br/>";//[^\u4E00-\u9FA5]
			Pattern namePattern = Pattern.compile(nameRegex);
			Matcher nameMatch = namePattern.matcher(tdHtml);
			String name = "";
			while(nameMatch.find()){
				name = nameMatch.group().replace("<br/>", "");
				break;
			}	
			
			MArea area = new MArea();
			area.setAreaCode(code);
			area.setAreaName(name);
			area.setParentCode(code);
			area.setAreaLevel("1"); //省份是一级（1）
			area.setAreaId(IDFactory.getId("meileche", "area"));
			this.addArea(area);
			
		}
		
		return list;
	}
	
	/**
	 * 解析234级城市代码和城市名称
	 * @param urlAddr url地址
	 * @param level 2、3、4级
	 * @param parentCode 父亲城市代码
	 * @return 返回子城市代码路径和代码
	 * @throws Exception 异常
	 */
	public List<String> parseCityAndCode(String urlAddr, String level, String parentCode) throws Exception{
		List<String> list = new LinkedList<String>();
		String htmlContent = "";
		try{
			htmlContent = this.get_www_stats_gov_cn_HtmlContent(urlAddr);
		}
		catch(Exception e){
			htmlContent = this.get_www_stats_gov_cn_HtmlContent(urlAddr);
			MLLogFactory.exceptionLog("url:" + urlAddr + "---------parentCode:" + parentCode + " -----------level:" + level, "抓取数据失败", e.getMessage(), "localhost");
		}
		String cityAndCodeRegex = "<td><a href='\\d{0,}/\\d{0,}.html'>\\d{0,}</a></td><td><a href='\\d{0,}/\\d{0,}.html'>[\u4e00-\u9fa5]{0,}</a></td>";
		Pattern pattern = Pattern.compile(cityAndCodeRegex);
		Matcher match = pattern.matcher(htmlContent);

		while(match.find()){
			String tdContent = match.group();
			//<td><a href='11/1101.html'>110100000000</a></td><td><a href='11/1101.html'>市辖区</a></td>

			//code 
			String codeRegex = "\\d{1,}</a>";
			Pattern codePattern = Pattern.compile(codeRegex);
			Matcher codeMatch = codePattern.matcher(tdContent);
			String code = "";
			while(codeMatch.find()){
				code = codeMatch.group().replace("</a>", "");
				break;
			}
			
			
			
			//url
			String urlRegex = "\\d{0,}/\\d{0,}.html";
			Pattern urlPattern = Pattern.compile(urlRegex);
			Matcher urlMatch = urlPattern.matcher(tdContent);
			while(urlMatch.find()){
				String thirdUrl = urlMatch.group();
				list.add(thirdUrl + "^" + code);
				break;
			}
			
			//name
			String name = "";
			String nameRegex = "[\u4e00-\u9fa5]{1,}</a>";
			Pattern namePattern = Pattern.compile(nameRegex);
			Matcher nameMatch = namePattern.matcher(tdContent);
			while(nameMatch.find()){
				name = nameMatch.group().replace("</a>", "");
				break;
			}
			
			MArea area = new MArea();
			area.setAreaCode(code);
			area.setAreaName(name);
			area.setParentCode(parentCode);
			area.setAreaLevel(level); //城市是二级（2）
			area.setAreaId(IDFactory.getId("meileche", "area"));
			this.addArea(area);
			
		}
		
		return list;
	}
	
	/**
	 * 获取第五级行政区域数据
	 * @param urlAddr 行政区域数据
	 * @param parentCode 父亲代码
	 * @return 返回空list(null)
	 * @throws Exception 异常
	 */
	public List<String> parseCountryAndCode(String urlAddr, String parentCode) throws Exception{
		List<String> list = null;
		String htmlContent = "";
		try{
			htmlContent = this.get_www_stats_gov_cn_HtmlContent(urlAddr);
		}
		catch(Exception e){
			htmlContent = this.get_www_stats_gov_cn_HtmlContent(urlAddr);
			MLLogFactory.exceptionLog(urlAddr + "---------" + parentCode, "抓取数据失败", e.getMessage(), "localhost");
		}
		String regex = "<td>\\d{0,}</td><td>\\d{0,}</td><td>[\u4e00-\u9fa5]{1,}</td>";
		Pattern pattern = Pattern.compile(regex);
		Matcher match = pattern.matcher(htmlContent);
		while(match.find()){
			//<td>110101001001</td><td>111</td><td>多福巷社区居委会</td>
			//无Url
			//code 
			String tdContent = match.group();
			String codeRegex = "\\d{4,}</td>";
			Pattern codePattern = Pattern.compile(codeRegex);
			Matcher codeMatch = codePattern.matcher(tdContent);
			String code = "";
			while(codeMatch.find()){
				code = codeMatch.group().replace("</td>", "");
				break;
			}
			
			String nameRegex = "[\u4e00-\u9fa5]{1,}</td>";
			Pattern namePattern = Pattern.compile(nameRegex);
			Matcher nameMatch = namePattern.matcher(tdContent);
			String name = "";
			while(nameMatch.find()){
				name = nameMatch.group().replace("</td>", "");
				break;
			}
			
			MArea area = new MArea();
			area.setAreaCode(code);
			area.setAreaName(name);
			area.setParentCode(parentCode);
			area.setAreaLevel("5"); //居委会是五级（5）
			area.setAreaId(IDFactory.getId("meileche", "area"));
			this.addArea(area);
		}
		
		return list;
	}
	
	public void crawlChinaArea(String year) throws Exception{
		String initUrl = this.statsUrl + year + "/";
		//一级
		List<String> urlList = this.parseProvinceAndCode(initUrl);
		//二级
		for (String url2 : urlList) {
			String[] url2Arr = url2.split("\\^");
			String secondUrl = initUrl + url2Arr[0];
			//三级
			List<String> secondUrlList = this.parseCityAndCode(secondUrl, "2", url2Arr[1]);
			for (String url3 : secondUrlList) {
				String[] url3Arr = url3.split("\\^");
				String nextUrlPre1 = url3Arr[0].split("\\/")[0];
				String thirdUrl = initUrl + url3Arr[0];
				//四级
				List<String> thirdUrlList = this.parseCityAndCode(thirdUrl, "3", url3Arr[1]);//返回三级地区的链接
				for (String url4 : thirdUrlList) {
					String[] url4Arr = url4.split("\\^");
					String fourUrl = initUrl +  nextUrlPre1 + "/" + url4Arr[0];
					String nextUrlPre2 = url4Arr[0].split("\\/")[0];
					List<String> fiveUrlList = this.parseCityAndCode(fourUrl, "4", url4Arr[1]);
					for (String url5 : fiveUrlList) {
						String[] url5Arr = url5.split("\\^");
						String fiveUrl = initUrl +  nextUrlPre1 + "/" + nextUrlPre2 + "/" + url5Arr[0];
						this.parseCountryAndCode(fiveUrl, url5Arr[1]);
					}
 				}
			}
		}
	}
	
	public static void main(String[] args) {
		MeilecheChinaAreaCrawler handler = new MeilecheChinaAreaCrawler();
		String year = "2013";
		try {
			handler.crawlChinaArea(year);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	/**
	 * 上一个区域节点的值
	 */
	private MArea preArea = null;
	
	
	public void addArea(MArea area) throws Exception{
		BArea handler = new BArea();
		if(!handler.isExistAreaByCode(area.getAreaCode())){
			if(handler.addArea(area) > 0){
				preArea = area;
				noScanAreaList.add(area);
				isAddScanArea = false;
			}
			
			if(isAddScanArea){
				lastScanArea = area;
			}
		}
	}
}
